
import scrapy

from scrapy.http import Request

from SinanewsSpider.items import SinanewsspiderItem

class SinanewsSpider(scrapy.Spider):
    # name = "SinanewsSpider"
    # start_url = []
    # def __init__(self):
    #     self.start_url = ["http://roll.news.sina.com.cn/news/gnxw/gdxw1/index.shtml"]
    #
    # def parse(self, response):
    #     for url in response.xpath('//ul/li/a/@href').extract():
    #         yield scrapy.Request(url, callback=self.parse_detail)
    #
    #     nextLink = []
    #     # 下一页
    #     nextLink = response.xpath('//div[@class="pagebox"]/span[last()-1]/a/@href').extract()
    #     if nextLink:
    #         nextLink = nextLink[0]
    #         # 获取下一个跳转的页面
    #         nextPage = nextLink.split('./')[1]
    #         yield Request("http://roll.news.sina.com.cn/news/gnxw/gdxw1/" + nextPage, callback = self.parse)
    #
    # def parse_detail(self,response):
    #     item = SinanewsSpider()
    #     item['title'] = response.xpath('//h1[@class="main-title"]/text()').extract()[0]
    #     content = ''
    #     for con in response.xpath('//div[@id="article"]/p/text()').extract():
    #         content = content + con
    #     item['content'] = content
    #
    #     item['pubtime'] = response.xpath('//span[@class="date"]/text()').extract()[0]
    #     imageUrl = ''
    #     for img in response.xpath('//div[@id="article"]/div[@class="img_wrapper"]/img/@src').extract():
    #         imageurl = imageurl + img + '|'
    #     item['imageUrl'] = imageurl
    #     item['Url'] = response.url
    #     yield item

    name = "SinanewsSpider"
    start_urls = []

    def __init__(self):
        self.start_urls = ["http://roll.news.sina.com.cn/news/gnxw/gdxw1/index.shtml"]

    def parse(self, response):
        for url in response.xpath('//ul/li/a/@href').extract():
            yield scrapy.Request(url, callback=self.parse_detail)

        nextLink = []
        nextLink = response.xpath('//div[@class="pagebox"]/span[last()-1]/a/@href').extract()
        if nextLink:
            nextLink = nextLink[0]
            nextpage = nextLink.split('./')[1]
            yield Request("http://roll.news.sina.com.cn/news/gnxw/gdxw1/" + nextpage, callback=self.parse)

    def parse_detail(self, response):
        item = SinanewsspiderItem()
        item['title'] = response.xpath('//h1[@class="main-title"]/text()').extract()[0]
        content = ''
        for con in response.xpath('//div[@id="article"]/p/text()').extract():
            content = content + con
        item['content'] = content
        item['pubtime'] = response.xpath('//span[@class="date"]/text()').extract()[0]
        imageurl = ''
        for img in response.xpath('//div[@id="article"]/div[@class="img_wrapper"]/img/@src').extract():
            imageurl = imageurl + img + '|'
        item['imageUrl'] = imageurl
        item['Url'] = response.url
        yield item

